print "Yes, let's do it.20130413"

outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/rangesForEachCellWithEvenValueFor100KQueries_1_10%"
outputFileHandler = open(outputFileName,"w")

freqWithMetaInfoForEachCellDict = {}

NUM_OF_RANGES = 5

# column0: cellKey
# column1: cellKey Value
COLUMN_INDEX_TO_COMPUTE = 1

# First Pass: compute the total freq
# variables init
totalFreqForMeToCompute = 0
rangeGapForCurrentFreq = 0
currentFreq = -1
currentSmallRangeID = -1

inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/100KQueries_1_10%_without_query_terms_sortedByMyOwnWay"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    currentLineCellKey = lineElements[0]
    currentLineCellKeyElements = currentLineCellKey.strip().split("_")
    currentLineFreq = int( lineElements[COLUMN_INDEX_TO_COMPUTE] )
    
    if int(currentLineCellKeyElements[0]) != currentFreq:
        rangeGapForCurrentFreq = int( totalFreqForMeToCompute/NUM_OF_RANGES )
        
        print "currentFreq:",currentFreq
        print "totalFreqForMeToCompute:",totalFreqForMeToCompute
        print "rangeGapForCurrentFreq:",rangeGapForCurrentFreq
        print 
        
        if currentFreq != -1 and currentFreq not in freqWithMetaInfoForEachCellDict:
            freqWithMetaInfoForEachCellDict[currentFreq] = (rangeGapForCurrentFreq,totalFreqForMeToCompute)
        
        totalFreqForMeToCompute = 0
        currentFreq = int(currentLineCellKeyElements[0])
        currentSmallRangeID = currentLineCellKeyElements[1]
        
        totalFreqForMeToCompute += currentLineFreq
    else:
        totalFreqForMeToCompute += currentLineFreq
inputFileHandler.close()

print "currentFreq:",currentFreq
print "totalFreqForMeToCompute:",totalFreqForMeToCompute
print "rangeGapForCurrentFreq:",rangeGapForCurrentFreq
print

if currentFreq != -1 and currentFreq not in freqWithMetaInfoForEachCellDict:
    freqWithMetaInfoForEachCellDict[currentFreq] = (rangeGapForCurrentFreq,totalFreqForMeToCompute)
    
print "len(freqWithMetaInfoForEachCellDict):",len(freqWithMetaInfoForEachCellDict)




currentFreq = -1
currentSmallRangeID = -1
currentRangeSet = 1
currentThreshold = -1
rangeGap = -1
lowerBoundSmallRangeID = -1
upperBoundSmallRangeID = -1

previousTotalFreqForMeToCompute = 0
previousContinusTotalFreqForMeToCompute = 0
totalFreqForMeToCompute = 0
# Second Pass: design the ranges for each cell
inputFileHandlerAgain = open(inputFileName,"r")
for index,line in enumerate( inputFileHandlerAgain.readlines() ):
    lineElements = line.strip().split(" ")
    currentLineCellKey = lineElements[0]
    currentLineCellKeyElements = currentLineCellKey.strip().split("_")
    currentLineFreq = int( lineElements[COLUMN_INDEX_TO_COMPUTE] )
    
    if int(currentLineCellKeyElements[0]) != currentFreq:
        numOfQueryTermsInThisRange = totalFreqForMeToCompute - previousTotalFreqForMeToCompute
        upperBoundSmallRangeID = "999"
        if int(lowerBoundSmallRangeID) == 0:
            print "[",lowerBoundSmallRangeID,",",upperBoundSmallRangeID,"]",":",numOfQueryTermsInThisRange
            outputLine = str(lowerBoundSmallRangeID) + " " + str(upperBoundSmallRangeID) + " " + str(numOfQueryTermsInThisRange) + "\n"
            outputFileHandler.write(outputLine)
        else:
            print "[",str( int(lowerBoundSmallRangeID) + 1 ),",",upperBoundSmallRangeID,"]",":",numOfQueryTermsInThisRange
            outputLine = str( int(lowerBoundSmallRangeID) + 1 ) + " " + str(upperBoundSmallRangeID) + " " + str(numOfQueryTermsInThisRange) + "\n"
            outputFileHandler.write(outputLine)            
            
        lowerBoundSmallRangeID = upperBoundSmallRangeID
        previousTotalFreqForMeToCompute = totalFreqForMeToCompute

        
        print "totalFreqForMeToCompute:",totalFreqForMeToCompute
        print "currentThreshold:",currentThreshold 
        print
        outputFileHandler.write("\n")
        
        previousContinusTotalFreqForMeToCompute = totalFreqForMeToCompute
        totalFreqForMeToCompute = 0
        previousTotalFreqForMeToCompute = 0
        currentRangeSet = 1
        currentFreq = int(currentLineCellKeyElements[0])
        currentSmallRangeID = currentLineCellKeyElements[1]
        
        (rangeGap,TOTAL_VALUE_FOR_DIVIDING) = freqWithMetaInfoForEachCellDict[currentFreq]
        currentThreshold = currentRangeSet * rangeGap
        
        lowerBoundSmallRangeID = currentSmallRangeID
        previousContinusTotalFreqForMeToCompute = totalFreqForMeToCompute
        totalFreqForMeToCompute += currentLineFreq
        print "currentFreq:",currentFreq
        print "average rangeGap:",rangeGap
        outputFileHandler.write(str(currentFreq) + " " + str(TOTAL_VALUE_FOR_DIVIDING) + " " + str(NUM_OF_RANGES) + " " + str(rangeGap) + " " + "\n")
    else:
        previousContinusTotalFreqForMeToCompute = totalFreqForMeToCompute
        totalFreqForMeToCompute += currentLineFreq
    
    
    
    if totalFreqForMeToCompute >= currentThreshold and currentRangeSet != 5:
        positiveDistance = totalFreqForMeToCompute - currentThreshold
        negativeDistance = currentThreshold - previousContinusTotalFreqForMeToCompute
        # for debug only
        # print "positive distance:", positiveDistance
        # print "negative distance:", negativeDistance
        
        if positiveDistance > negativeDistance:
            # for debug only
            # print "need to back off to the previous one"
            
            # reset the new upperBoundSmallRangeID here by decreasing one
            upperBoundSmallRangeID = str( int( currentLineCellKeyElements[1] ) - 1 ) 
            currentRangeSet += 1
            currentThreshold = currentRangeSet * rangeGap
            numOfQueryTermsInThisRange = previousContinusTotalFreqForMeToCompute - previousTotalFreqForMeToCompute
            
            if int(lowerBoundSmallRangeID) == 0:
                print "[",lowerBoundSmallRangeID,",",upperBoundSmallRangeID,"]",":",numOfQueryTermsInThisRange
                outputLine = str( lowerBoundSmallRangeID ) + " " + str( upperBoundSmallRangeID ) + " " + str( numOfQueryTermsInThisRange ) + "\n"
                outputFileHandler.write(outputLine)
            else:
                print "[",str( int(lowerBoundSmallRangeID) + 1 ),",",upperBoundSmallRangeID,"]",":",numOfQueryTermsInThisRange
                outputLine = str( int(lowerBoundSmallRangeID) + 1 ) + " " + str( upperBoundSmallRangeID ) + " " + str( numOfQueryTermsInThisRange ) + "\n"
                outputFileHandler.write(outputLine) 
                               
            lowerBoundSmallRangeID = upperBoundSmallRangeID
            previousTotalFreqForMeToCompute = previousContinusTotalFreqForMeToCompute
        else:
            upperBoundSmallRangeID = currentLineCellKeyElements[1]
            currentRangeSet += 1
            currentThreshold = currentRangeSet * rangeGap
            numOfQueryTermsInThisRange = totalFreqForMeToCompute - previousTotalFreqForMeToCompute
            
            if int(lowerBoundSmallRangeID) == 0:
                print "[",lowerBoundSmallRangeID,",",upperBoundSmallRangeID,"]",":",numOfQueryTermsInThisRange
                outputLine = str( lowerBoundSmallRangeID ) + " " + str( upperBoundSmallRangeID ) + " " + str( numOfQueryTermsInThisRange ) + "\n"
                outputFileHandler.write(outputLine)
            else:
                print "[",str( int(lowerBoundSmallRangeID) + 1 ),",",upperBoundSmallRangeID,"]",":",numOfQueryTermsInThisRange
                outputLine = str( int(lowerBoundSmallRangeID) + 1 ) + " " + str( upperBoundSmallRangeID ) + " " + str( numOfQueryTermsInThisRange ) + "\n"
                outputFileHandler.write(outputLine)
            lowerBoundSmallRangeID = upperBoundSmallRangeID
            previousTotalFreqForMeToCompute = totalFreqForMeToCompute

numOfQueryTermsInThisRange = totalFreqForMeToCompute - previousTotalFreqForMeToCompute
upperBoundSmallRangeID = "999"
if int(lowerBoundSmallRangeID) == 0:
    print "[",lowerBoundSmallRangeID,",",upperBoundSmallRangeID,"]",":",numOfQueryTermsInThisRange
    outputLine = str( lowerBoundSmallRangeID ) + " " + str( upperBoundSmallRangeID ) + " " + str( numOfQueryTermsInThisRange ) + "\n"
    outputFileHandler.write(outputLine)
else:
    print "[",str( int(lowerBoundSmallRangeID) + 1 ),",",upperBoundSmallRangeID,"]",":",numOfQueryTermsInThisRange
    outputLine = str( int(lowerBoundSmallRangeID) + 1 ) + " " + str( upperBoundSmallRangeID ) + " " + str( numOfQueryTermsInThisRange ) + "\n"
    outputFileHandler.write(outputLine)
       
print "totalFreqForMeToCompute:",totalFreqForMeToCompute
print "currentThreshold:",currentThreshold
print
outputFileHandler.write("\n")

inputFileHandlerAgain.close()
outputFileHandler.close()



'''
smallRangeIDAndZeroFreqDict = {}
inputAuxFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/eachSmallRangeInfo20130412_sortedBySmallRangeID"
inputAuxFileHandler = open(inputAuxFileName,"r")
for line in inputAuxFileHandler.readlines():
    lineElements = line.strip().split(" ")
    smallRangeID = lineElements[0]
    freq = int(lineElements[3])
    if smallRangeID not in smallRangeIDAndZeroFreqDict:
        smallRangeIDAndZeroFreqDict[smallRangeID] = freq
    else:
        print "Unexpected Behaviour"
        exit(1)

print "len(smallRangeIDAndZeroFreqDict):",len(smallRangeIDAndZeroFreqDict)
inputAuxFileHandler.close()

inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/freqOfFreqInQueries_head_85K_0_85%_2D_without_query_terms_sortedByMyOwnWay"
inputFileHandler = open(inputFileName,"r")

outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/freqOfFreqInQueries_head_85K_0_85%_2D_without_query_terms_fix_zerop_freq_sortedByMyOwnWay"
outputFileHandler = open(outputFileName,"w")

for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    if lineElements[0].startswith("0_"):
        # need to fix the corresponding value
        smallRangeID = lineElements[0].split("_")[1]
        cellKey = "0_" + smallRangeID
        newZeroFreqValue = smallRangeIDAndZeroFreqDict[smallRangeID]
        
        outputFileHandler.write(cellKey + " " + str(newZeroFreqValue) + "\n")
    else:
        outputFileHandler.write(line)

inputFileHandler.close()
outputFileHandler.close()
'''